# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html

from scrapy import signals

# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter

# from utils.proxies_manage import get_random_proxy


class QidianCrawlerSpiderMiddleware:
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the spider middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_spider_input(self, response, spider):
        # Called for each response that goes through the spider
        # middleware and into the spider.

        # Should return None or raise an exception.
        return None

    def process_spider_output(self, response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.

        # Must return an iterable of Request, or item objects.
        for i in result:
            yield i

    def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Request or item objects.
        pass

    def process_start_requests(self, start_requests, spider):
        # Called with the start requests of the spider, and works
        # similarly to the process_spider_output() method, except
        # that it doesn’t have a response associated.

        # Must return only requests (not items).
        for r in start_requests:
            yield r

    def spider_opened(self, spider):
        spider.logger.info("Spider opened: %s" % spider.name)


class QidianCrawlerDownloaderMiddleware:
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.

        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called
        # cookies = {
        #     'e1': '%7B%22l6%22%3A%22%22%2C%22l7%22%3A%22%22%2C%22l1%22%3A1%2C%22l3%22%3A%22%22%2C%22pid%22%3A%22qd_P_free%22%2C%22eid%22%3A%22qd_A01%22%7D',
        #     'e2': '%7B%22l6%22%3A%22%22%2C%22l7%22%3A%22%22%2C%22l1%22%3A%22%22%2C%22l3%22%3A%22%22%2C%22pid%22%3A%22qd_P_free%22%2C%22eid%22%3A%22%22%7D',
        #     'e1': '%7B%22l6%22%3A%22%22%2C%22l7%22%3A%22%22%2C%22l1%22%3A4%2C%22l3%22%3A%22%22%2C%22pid%22%3A%22qd_P_limitfree%22%2C%22eid%22%3A%22qd_E01%22%7D',
        #     'e2': '%7B%22l6%22%3A%22%22%2C%22l7%22%3A%22%22%2C%22l1%22%3A3%2C%22l3%22%3A%22%22%2C%22pid%22%3A%22qd_P_limitfree%22%2C%22eid%22%3A%22qd_A15%22%7D',
        #     'traffic_utm_referer': 'https%3A//www.baidu.com/link',
        #     'supportwebp': 'true',
        #     '_gid': 'GA1.2.2080151495.1751942694',
        #     '_csrfToken': 'jT2D3OP5kzrnepqMXXUIFygYsLeond1sz3DPN04O',
        #     'newstatisticUUID': '1751974335_707196724',
        #     'fu': '252044939',
        #     'Hm_lvt_f00f67093ce2f38f215010b699629083': '1751937733,1751974335',
        #     'HMACCOUNT': '78EB726CA5B89125',
        #     'e1': '%7B%22l6%22%3A%22%22%2C%22l7%22%3A%22%22%2C%22l1%22%3A3%2C%22l3%22%3A%22%22%2C%22pid%22%3A%22qd_p_qidian%22%2C%22eid%22%3A%22qd_A18%22%7D',
        #     'e2': '%7B%22l6%22%3A%22%22%2C%22l7%22%3A%22%22%2C%22l1%22%3A3%2C%22l3%22%3A%22%22%2C%22pid%22%3A%22qd_p_qidian%22%2C%22eid%22%3A%22qd_A15%22%7D',
        #     'Hm_lpvt_f00f67093ce2f38f215010b699629083': '1751975792',
        #     '_gat_gtag_UA_199934072_2': '1',
        #     '_ga_FZMMH98S83': 'GS2.1.s1751974335$o4$g1$t1751975792$j60$l0$h0',
        #     '_ga': 'GA1.1.1220863926.1751937733',
        #     '_ga_PFYW0QLV3P': 'GS2.1.s1751974335$o4$g1$t1751975792$j60$l0$h0',
        #     'w_tsfp': 'ltvuV0MF2utBvS0Q7Kvgk06mFz8ucTw4h0wpEaR0f5thQLErU5mA0I99u8P2OHHY68xnvd7DsZoyJTLYCJI3dwMSR8SVINtE2l/CwNIgjohBCRRkRJLVXQIZJrgmvjlEe3hCNxS00jA8eIUd379yilkMsyN1zap3TO14fstJ019E6KDQmI5uDW3HlFWQRzaLbjcMcuqPr6g18L5a5TqI4FP6Kgl2UrwR0EaU1nsaXy1250e9JuxaMEquIZiuSqA=',
        # }
        #
        # headers = {
        #     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
        #     'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
        #     'Cache-Control': 'max-age=0',
        #     'Connection': 'keep-alive',
        #     'Referer': 'https://www.qidian.com/free/all/',
        #     'Sec-Fetch-Dest': 'document',
        #     'Sec-Fetch-Mode': 'navigate',
        #     'Sec-Fetch-Site': 'same-origin',
        #     'Sec-Fetch-User': '?1',
        #     'Upgrade-Insecure-Requests': '1',
        #     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0',
        #     'sec-ch-ua': '"Not)A;Brand";v="8", "Chromium";v="138", "Microsoft Edge";v="138"',
        #     'sec-ch-ua-mobile': '?0',
        #     'sec-ch-ua-platform': '"Windows"',
        #     # 'Cookie': 'e1=%7B%22l6%22%3A%22%22%2C%22l7%22%3A%22%22%2C%22l1%22%3A1%2C%22l3%22%3A%22%22%2C%22pid%22%3A%22qd_P_free%22%2C%22eid%22%3A%22qd_A01%22%7D; e2=%7B%22l6%22%3A%22%22%2C%22l7%22%3A%22%22%2C%22l1%22%3A%22%22%2C%22l3%22%3A%22%22%2C%22pid%22%3A%22qd_P_free%22%2C%22eid%22%3A%22%22%7D; e1=%7B%22l6%22%3A%22%22%2C%22l7%22%3A%22%22%2C%22l1%22%3A4%2C%22l3%22%3A%22%22%2C%22pid%22%3A%22qd_P_limitfree%22%2C%22eid%22%3A%22qd_E01%22%7D; e2=%7B%22l6%22%3A%22%22%2C%22l7%22%3A%22%22%2C%22l1%22%3A3%2C%22l3%22%3A%22%22%2C%22pid%22%3A%22qd_P_limitfree%22%2C%22eid%22%3A%22qd_A15%22%7D; traffic_utm_referer=https%3A//www.baidu.com/link; supportwebp=true; _gid=GA1.2.2080151495.1751942694; _csrfToken=jT2D3OP5kzrnepqMXXUIFygYsLeond1sz3DPN04O; newstatisticUUID=1751974335_707196724; fu=252044939; Hm_lvt_f00f67093ce2f38f215010b699629083=1751937733,1751974335; HMACCOUNT=78EB726CA5B89125; e1=%7B%22l6%22%3A%22%22%2C%22l7%22%3A%22%22%2C%22l1%22%3A3%2C%22l3%22%3A%22%22%2C%22pid%22%3A%22qd_p_qidian%22%2C%22eid%22%3A%22qd_A18%22%7D; e2=%7B%22l6%22%3A%22%22%2C%22l7%22%3A%22%22%2C%22l1%22%3A3%2C%22l3%22%3A%22%22%2C%22pid%22%3A%22qd_p_qidian%22%2C%22eid%22%3A%22qd_A15%22%7D; Hm_lpvt_f00f67093ce2f38f215010b699629083=1751975792; _gat_gtag_UA_199934072_2=1; _ga_FZMMH98S83=GS2.1.s1751974335$o4$g1$t1751975792$j60$l0$h0; _ga=GA1.1.1220863926.1751937733; _ga_PFYW0QLV3P=GS2.1.s1751974335$o4$g1$t1751975792$j60$l0$h0; w_tsfp=ltvuV0MF2utBvS0Q7Kvgk06mFz8ucTw4h0wpEaR0f5thQLErU5mA0I99u8P2OHHY68xnvd7DsZoyJTLYCJI3dwMSR8SVINtE2l/CwNIgjohBCRRkRJLVXQIZJrgmvjlEe3hCNxS00jA8eIUd379yilkMsyN1zap3TO14fstJ019E6KDQmI5uDW3HlFWQRzaLbjcMcuqPr6g18L5a5TqI4FP6Kgl2UrwR0EaU1nsaXy1250e9JuxaMEquIZiuSqA=',
        # }
        cookies = {
            'supportwebp': 'true',
            '_gid': 'GA1.2.2080151495.1751942694',
            '_csrfToken': 'jT2D3OP5kzrnepqMXXUIFygYsLeond1sz3DPN04O',
            'newstatisticUUID': '1751974335_707196724',
            'fu': '252044939',
            'Hm_lvt_f00f67093ce2f38f215010b699629083': '1751937733,1751974335',
            'HMACCOUNT': '78EB726CA5B89125',
            'e1': '%7B%22l6%22%3A%22%22%2C%22l7%22%3A%22%22%2C%22l1%22%3A3%2C%22l3%22%3A%22%22%2C%22pid%22%3A%22qd_p_qidian%22%2C%22eid%22%3A%22qd_A17%22%7D',
            'e2': '%7B%22l6%22%3A%22%22%2C%22l7%22%3A%22%22%2C%22l1%22%3A3%2C%22l3%22%3A%22%22%2C%22pid%22%3A%22qd_p_qidian%22%2C%22eid%22%3A%22qd_A18%22%7D',
            'supportWebp': 'true',
            'traffic_search_engine': '',
            'se_ref': '',
            'x-waf-captcha-referer': '',
            'traffic_utm_referer': 'https%3A%2F%2Fwww.baidu.com%2Flink',
            'Hm_lpvt_f00f67093ce2f38f215010b699629083': '1751984913',
            '_gat_gtag_UA_199934072_2': '1',
            '_ga_FZMMH98S83': 'GS2.1.s1751974335$o4$g1$t1751984913$j60$l0$h0',
            '_ga': 'GA1.1.1220863926.1751937733',
            '_ga_PFYW0QLV3P': 'GS2.1.s1751974335$o4$g1$t1751984913$j60$l0$h0',
            'w_tsfp': 'ltvuV0MF2utBvS0Q7KvgnE+nEjwhdD44h0wpEaR0f5thQLErU5mA0I9yusLzNXTW5cxnvd7DsZoyJTLYCJI3dwNHRcuXJ9pAjAzEk4Um3IgUBUZmR5iNCwMfI+px6mUTKnhCNxS00jA8eIUd379yilkMsyN1zap3TO14fstJ019E6KDQmI5uDW3HlFWQRzaLbjcMcuqPr6g18L5a5TvfsFKqeF8lAegU2UGQ0CBKCHx2txiyIOsONRWkJ5+oSqA=',
        }

        headers = {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
            'Cache-Control': 'max-age=0',
            'Connection': 'keep-alive',
            # 'Referer': 'https://www.qidian.com/chapter/1045165156/848398372/',
            'Sec-Fetch-Dest': 'document',
            'Sec-Fetch-Mode': 'navigate',
            'Sec-Fetch-Site': 'same-origin',
            'Sec-Fetch-User': '?1',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0',
            'sec-ch-ua': '"Not)A;Brand";v="8", "Chromium";v="138", "Microsoft Edge";v="138"',
            'sec-ch-ua-mobile': '?0',
            'sec-ch-ua-platform': '"Windows"',
            # 'Cookie': 'supportwebp=true; _gid=GA1.2.2080151495.1751942694; _csrfToken=jT2D3OP5kzrnepqMXXUIFygYsLeond1sz3DPN04O; newstatisticUUID=1751974335_707196724; fu=252044939; Hm_lvt_f00f67093ce2f38f215010b699629083=1751937733,1751974335; HMACCOUNT=78EB726CA5B89125; e1=%7B%22l6%22%3A%22%22%2C%22l7%22%3A%22%22%2C%22l1%22%3A3%2C%22l3%22%3A%22%22%2C%22pid%22%3A%22qd_p_qidian%22%2C%22eid%22%3A%22qd_A17%22%7D; e2=%7B%22l6%22%3A%22%22%2C%22l7%22%3A%22%22%2C%22l1%22%3A3%2C%22l3%22%3A%22%22%2C%22pid%22%3A%22qd_p_qidian%22%2C%22eid%22%3A%22qd_A18%22%7D; supportWebp=true; traffic_search_engine=; se_ref=; x-waf-captcha-referer=; traffic_utm_referer=https%3A%2F%2Fwww.baidu.com%2Flink; Hm_lpvt_f00f67093ce2f38f215010b699629083=1751984913; _gat_gtag_UA_199934072_2=1; _ga_FZMMH98S83=GS2.1.s1751974335$o4$g1$t1751984913$j60$l0$h0; _ga=GA1.1.1220863926.1751937733; _ga_PFYW0QLV3P=GS2.1.s1751974335$o4$g1$t1751984913$j60$l0$h0; w_tsfp=ltvuV0MF2utBvS0Q7KvgnE+nEjwhdD44h0wpEaR0f5thQLErU5mA0I9yusLzNXTW5cxnvd7DsZoyJTLYCJI3dwNHRcuXJ9pAjAzEk4Um3IgUBUZmR5iNCwMfI+px6mUTKnhCNxS00jA8eIUd379yilkMsyN1zap3TO14fstJ019E6KDQmI5uDW3HlFWQRzaLbjcMcuqPr6g18L5a5TvfsFKqeF8lAegU2UGQ0CBKCHx2txiyIOsONRWkJ5+oSqA=',
        }
        for key, value in cookies.items():
            request.cookies[key] = value
        for key, value in headers.items():
            request.headers[key] = value
        return None

    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.

        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest
        return response

    def process_exception(self, request, exception, spider):
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception
        # - return a Response object: stops process_exception() chain
        # - return a Request object: stops process_exception() chain
        pass

    def spider_opened(self, spider):
        spider.logger.info("Spider opened: %s" % spider.name)


from scrapy import signals


# class ProxyMiddleware:
#     def process_request(self, request, spider):
#         proxy_url = get_random_proxy()
#         request.meta['proxy'] = proxy_url
#         spider.logger.info(f"Using proxy: {proxy_url}")
#         return None  # 继续正常处理
#
#     def process_response(self, request, response, spider):
#         # 检查是否返回异常状态码（如429 Too Many Requests）
#         if response.status >= 400:
#             spider.logger.warning(f"Received status code {response.status}, retrying with a new proxy...")
#             new_proxy = get_random_proxy()
#             request.meta['proxy'] = new_proxy
#             spider.logger.info(f"Retrying with proxy: {new_proxy}")
#             return request.copy()  # 重新发起请求
#         return response
